#include "Host.hpp"
#include "Http.hpp"
#include "Page.hpp"
#include "util/String.hpp"

#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>

#include <sstream>

#include <iostream>

using namespace std;

Host_Elem
Host::GetElem(const string& crHostName)
{
    Host_Elem result;
    map<const string, Host_Elem>::iterator it;
    it = mHostMap.find(crHostName);
    if (it != mHostMap.end()) {
        result = it->second;
    } else {
        result = AddHostElem(crHostName);
    }
    return result;
}

string
Host::GetIpByHost(const string& crHost)
{
    string result;

    struct hostent     *hp;     // Host entity
    hp = gethostbyname(crHost.c_str());

    if(hp == NULL) {
        return result;
    }

    struct in_addr in;

    bcopy (*(hp->h_addr_list), (caddr_t) & in, hp->h_length);
    char abuf[INET_ADDRSTRLEN];

    inet_ntop(AF_INET, (void *)&in, abuf, sizeof(abuf));

    result = abuf;
    return result;
}

vector<string>
Host::GetRobotRules(const string& crHost, const string& crIP)
{
    vector<string> robots_rules;
    Url url;
    Page page;

    url.mHost = crHost;
    url.mPath = "/robots.txt";
    url.mIP   = crIP;

    Http http;

    if (!http.FetchUrl(url, page)){
        cout << "\tNOTE: Can not get robots.txt in host: " << crHost << endl;
        return robots_rules;
    }

    string content = page.GetContent();

    // Parse robots.txt, should be vilify
    istringstream scin(content);

    string tmp;
    while (getline(scin, tmp)) {
        String::Str2Lower(tmp);
        if (tmp.substr(0, 13) == "user-agent: *" ||
            tmp.substr(0, 12) == "user-agent:*") {
            break;
        }
    }

    while (getline(scin, tmp)) {
        String::Str2Lower(tmp);
        // modified by liuyuan, avoid exceptions
        if (tmp.substr(0, 9) == "disallow:" && tmp.length() > 10) {
            istringstream strin(tmp.substr(10));
            string rule;
            strin >> rule;
            if (!rule.empty()) {
                robots_rules.push_back(rule);
            }
        } else if (tmp.empty()){
            continue;
        } else {
            break;
        }
    }
    return robots_rules;
}

Host_Elem
Host::AddHostElem(const string& crHostName)
{
    Host_Elem host_elem;
    host_elem.IP = GetIpByHost(crHostName);
    host_elem.RobotRules = GetRobotRules(crHostName, host_elem.IP);

    mHostMap[crHostName] = host_elem;
    return host_elem;
}
